# coding=utf-8
import json

from lxml import etree
import requests


class DigitalassetsSpider(object):
    """
    main Spider
    """

    def __init__(self):
        self.base_url = 'http://digitalassets.southafrica.net/'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/39.0.2171.71 Safari/537.36 '
        }

    def load_page(self):
        """
        加载页面
        :return:
        """
        response = requests.get(url=self.base_url, headers=self.headers)
        html = response.text
        self.load_category(html)

    def load_category(self, html):
        """
        首先加载类别信息
        :param html: html文档
        :return:
        """
        html = etree.HTML(html)  # xpath解析首先转换etree
        parent_tags = html.xpath('//div[contains(@id,"newMenu")]')  # 首先加载父类别
        categories = []
        image_subs = []
        footage_subs = []
        marketing_subs = []
        for parent_tag in parent_tags:
            category = {
                'parent_category': parent_tag.text
            }
            categories.append(category)

        image_sub_tags = html.xpath(
            '//div[@id="dvTree_Image"]/div[@id="sabMillerNav_TreeView_Images"]/table'
        )  # 第一个分类的子类别table节点 一共7个
        footage_sub_tags = html.xpath(
            '//div[@id="dvTree_footage"]/div[@id="sabMillerNav_TreeView_Footage"]/table'
        )  # 第二个分类的子类别table节点 一共2个
        marketing_sub_tags = html.xpath(
            '//div[@id="dvTree_Regional"]/div[@id="sabMillerNav_TreeView_Regional"]/table'
        )  # 第二个分类的子类别table节点 一共7个

        for image_sub_tag in image_sub_tags:
            _sub = image_sub_tag.xpath('./tr/td[2]/a[contains(@id,"sabMillerNav_TreeView_Imagest")]')  # 取出子类别内容
            _sub = _sub[0].text
            image_subs.append(_sub)

        for footage_sub_tag in footage_sub_tags:
            _sub = footage_sub_tag.xpath('./tr/td[2]/a[contains(@id,"sabMillerNav_TreeView_Footaget")]')  # 取出子类别内容
            _sub = _sub[0].text
            footage_subs.append(_sub)

        for marketing_sub_tag in marketing_sub_tags:
            _sub = marketing_sub_tag.xpath('./tr/td[2]/a[contains(@id,"sabMillerNav_TreeView_Regionalt")]')  # 获取子类别内容
            _sub = _sub[0].text
            marketing_subs.append(_sub)

        categories[0]['sub'] = json.dumps(image_subs)
        categories[1]['sub'] = json.dumps(footage_subs)
        categories[2]['sub'] = json.dumps(marketing_subs)

        with open('category.json', 'w') as f:
            f.write(json.dumps(categories))


if __name__ == '__main__':
    DigitalassetsSpider().load_page()
